
runProcess <- function() {
  library(XML)
  library(stringr)
  library(RCurl)
  rootInput <- 'C:/SalesInsight/dataFiles/Resumes/'
  rootOutput <- 'C:/SalesInsight/dataFiles/ControlFiles/'
  algoFile <- 'C:/SalesInsight/dataFiles/algoInfo.txt'
  
  compProdKey <- read.table(file=str_c(algoFile), quote='',header=FALSE, sep=',', col.names=c('algo','algoId'), colClasses=c('character','character') )
  
  for(p in 1:nrow(compProdKey)) {
    algo <- compProdKey$algo[p]
    algoId <- compProdKey$algoId[p]
    
    algoInputFolder<- str_c(rootInput,algoId)
    if(!file.exists(algoInputFolder)) {
      dir.create(algoInputFolder)
    }
    algoInputFolder<- str_c(algoInputFolder,'/')
    
    algoOutputFolder <- str_c(rootOutput,algoId)
    if(!file.exists(algoOutputFolder)) {
      dir.create(algoOutputFolder)
    }
    algoOutputFolder <- str_c(algoOutputFolder, '/')
    
    downloadResumes(algo, algoId, algoInputFolder, algoOutputFolder)
    ##try again failed resume links
    downloadFailedResumes(algoOutputFolder, algoId)
  }
}

downloadResumes <- function(compProdKey, algoId, algoInputFolder, algoOutputFolder) {
  logInfo(algoOutputFolder, algoId, str_c('Started processing resumes for -> ',algoId))
  logInfo(algoOutputFolder, algoId, str_c('Ran with algo -> ',compProdKey))
  
  linksFile <- str_c(algoOutputFolder,'_', algoId, 'links.txt')
  if(file.exists(linksFile)) {
    file.remove(linksFile)
  }

  getMonthBreakdown(compProdKey=compProdKey, start=1,end=480, incremeant=12, fileName=linksFile)
    
  processLinks <- read.table(file=linksFile, quote='',header=FALSE, sep=',', col.names=c('Link','Count'))

  for (y in 1:nrow(processLinks))  {
      link <- processLinks$Link[y]
      count <- processLinks$Count[y]
      start <- 0  
    for(i in 0:floor(count/10)) {
      tryCatch( { 
     start <- i* 10 + 1 
     queryStrResumes <- str_c('&start=',start)
     finalResumesUrl <- str_c(link, queryStrResumes)
     logInfo(algoOutputFolder, algoId, str_c('Now Processing Link -> ', finalResumesUrl))
     countResult <- htmlParse(finalResumesUrl)
     links <- xpathApply(countResult,'//a[substring(@href,1,3) = \'/r/\' or substring(@href,1,4) = \'/me/\']',xmlGetAttr,"href")
  
     for(j in 1:length(links)) {
       tryCatch( { 
       currentURL <- str_c('http://www.indeed.com',links[[j]])
       splitURL <- str_split(currentURL,'/')[[1]]
       fileId <- splitURL[length(splitURL)]
       fileName = str_c(algoInputFolder,fileId,'.txt')
       if(!file.exists(fileName)) {
         txt <- getURL(currentURL)
         fileConn<-file(fileName)
         writeLines(txt, fileConn)
         close(fileConn)
       } else {
         logInfo(algoOutputFolder, algoId, str_c('file already exists : ', fileId))
       }
     }, error = function(e) {
       logFailedResumeLInks(algoOutputFolder, algoId, currentURL)
     })
     }
      }, error = function(e) {
        logFailedLInks(algoOutputFolder, algoId, finalResumesUrl)
      })
    }

  }
  logInfo(algoOutputFolder, algoId, str_c('Done processing resumes for -> ',algoId))
}

getMonthBreakdown <- function(compProdKey, start, end, incremeant, fileName) {
  x <- incremeant
  urlInfoMain <- getResumeCount(compProdKey,str_c(toString(start),'-',toString(end)))
  if(urlInfoMain$count > 1000) {
    for(i in seq(start,end,by=x)) {
      urlInfo <- getResumeCount(compProdKey,str_c(toString(i),'-',toString(i+x)))
      if(urlInfo$count > 1000 && x != 1) {
        getMonthBreakdown(compProdKey,i,i+x,ceiling(x/2),fileName)
      } else if(urlInfo$count != 0) {
        write.table(col.names=FALSE, quote=FALSE, urlInfo,file=fileName,append=TRUE, sep=',' ,eol='\r\n', row.names=FALSE)
      }
    }
  } else {
    write.table(col.names=FALSE, quote=FALSE, urlInfoMain,file=fileName,append=TRUE, sep=',' ,eol='\r\n', row.names=FALSE)
  }
}

getResumeCount <- function(compProdKey, yearsOfExperience=NULL) {
  urlInfo <- data.frame(link=NULL,count=NULL)
  
  url <- 'http://www.indeed.com/resumes'
  if(is.null(yearsOfExperience)) {
    queryStr <- str_c('?q=',compProdKey)
  } else {
    queryStr <- str_c('?q=',compProdKey,'&rb=yoe:', yearsOfExperience)
  }
  
  finalUrl <- str_c(url, queryStr)
  numberOfResumes <- tryCatch({
    countResult <- htmlParse(finalUrl)
    countXML <- getNodeSet(countResult,'//div[@id=\'result_count\']')
    tempDF <- xmlToDataFrame(countXML)
    count <- as.numeric(str_replace(str_replace_all(xmlToDataFrame(countXML)$text,'[a-zA-Z]',''),',',''))
    numberOfResumes <- count[[1]]
  }, error = function(e) {
    numberOfResumes <- 0
  })
  
  urlInfo <- rbind(urlInfo,data.frame(link=finalUrl,count=numberOfResumes))
  return (urlInfo)
}

logInfo <- function(algoOutputFolder, algoId, message='no message passed') {
  infoFile <- str_c(algoOutputFolder,'_', algoId, 'info.txt')
  write(message, file=infoFile, append=TRUE, ncolumns=1)
}

logFailedResumeLInks <- function(algoOutputFolder, algoId, link='no link found') {
  errorResumeLinksFile <- str_c(algoOutputFolder,'_', algoId, 'FailedResumelinks.txt')
  write(link, file=errorResumeLinksFile, append=TRUE, ncolumns=1)
}

logFailedLInks <- function(algoOutputFolder, algoId, link='no link found') {
  errorLinksFile <- str_c(algoOutputFolder,'_', algoId, 'Failedlinks.txt')
  write(link, file=errorLinksFile, append=TRUE, ncolumns=1)
}

downloadFailedResumes <- function(algoOutputFolder, algoId) {
  errorLinksFile <- str_c(algoOutputFolder,'_', algoId, 'FailedResumelinks.txt')
  if(file.exists(errorLinksFile)) {
    links <- read.table(file=str_c(errorLinksFile), header=FALSE, sep=',', col.names=c('Link'), colClasses=c('character') )
    file.remove(errorLinksFile)
  
    for(j in 1:length(links)) {
      tryCatch( { 
        currentURL <- links$Link[j]
        splitURL <- str_split(currentURL,'/')[[1]]
        fileId <- splitURL[length(splitURL)]
        fileName = str_c(algoInputFolder,fileId,'.txt')
        if(!file.exists(fileName)) {
          txt <- getURL(currentURL)
          fileConn<-file(fileName)
          writeLines(txt, fileConn)
          close(fileConn)
        } else {
          print(str_c('file already exists: ', fileId))
        }
      }, error = function(e) {
        logFailedResumeLInks(algoOutputFolder,currentURL)
      })
    }
  }
}